{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# W3_冯炳驹_124298228"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第六步：调整学习率和Gamma\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# path to where the data lies\n",
    "dpath = './data/'\n",
    "train = pd.read_csv(dpath +\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEFCAYAAAD0cwBnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGmhJREFUeJzt3XuUXWWd5vFvpRJyIZV0jRQyjiCGdh6ZFjQGSGySMUIU\nwQtqd9vKZdHShkvTRLvphpYkKq6wWKLBTnBMnEAgELRZRhA7DhBahFymCRC5JAv8CRkv9IzaJeRS\nGJIQcuaPdx9zOqTO2ZvUqXNOneezVhZnv+fdu36brFVP3v3u/e6OUqmEmZlZXsMaXYCZmbUWB4eZ\nmRXi4DAzs0IcHGZmVoiDw8zMChne6ALqrbe3z7eNmZkV1NPT1dHfdx5xmJlZIQ4OMzMrxMFhZmaF\n1G2OQ1InsAQQUAIuAkYAK4Fnsm6LIuJ2STOBC4E9wLyIWClpNLAcOBzoA86LiF5JU4AFWd9VEXFV\nvc7BzMxerZ4jjg8BRMTJwBzgamAScF1ETM/+3C7pCGAWcDJwGnCNpJHAxcDGiJgG3JIdA2AxcBYw\nFZgsaWIdz8HMzPZTtxFHRHxP0sps803AVlJwSNKZpFHHZ4GTgHURsQvYJelZ4HhSMFyb7X83MFfS\nOGBkRGwmHeheYAbwWH91dHePYfjwzgE/PzOzdlXX23EjYo+kZcBHgT8F/gtwQ0RskDQb+ALwOLCt\nYrc+YDwwrqK9sm37fn0nVKthy5YdA3AmZmbtpaenq9/v6j45HhHnAf+VNN+xKiI2ZF/dCUwkBUFl\nhV2k0Ull+4HaKtvNzGyQ1C04JJ0r6XPZ5g5gL3CHpJOytlOBDcDDwDRJoySNB44FNgHrgDOyvqcD\nayJiO7Bb0jGSOkhzImvqdQ5mZvZq9bxUdQdwk6TVpLupPgs8B1wv6WXg18AFEbFd0kJSAAwDZkfE\nTkmLgGWS1gK7SRPikO7Oug3oJI1g1tfxHMzMbD8dQ/1FTl5yxKw1PHLZrEaXMOSdOH9h7r5ecsTM\nzAaMg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZ\nWSEODjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZm\nhTg4zMyskOH1OrCkTmAJIKAEXATsBG7OtjcBl0TEXkkzgQuBPcC8iFgpaTSwHDgc6APOi4heSVOA\nBVnfVRFxVb3OwczMXq2eI44PAUTEycAc4GrgOmBOREwDOoAzJR0BzAJOBk4DrpE0ErgY2Jj1vSU7\nBsBi4CxgKjBZ0sQ6noOZme2nbsEREd8DLsg23wRsBSYBD2ZtdwMzgJOAdRGxKyK2Ac8Cx5OC4Z7K\nvpLGASMjYnNElIB7s2OYmdkgqdulKoCI2CNpGfBR4E+B92a/8CFdfhoPjAO2Vex2oPbKtu379Z1Q\nrYbu7jEMH955kGdiZtb6enq6BuQ4dQ0OgIg4T9IVwHpgdMVXXaRRyPbsc7X2Wn37tWXLjoMp38xs\nyOjt7cvdt1rI1O1SlaRzJX0u29wB7AUelTQ9azsdWAM8DEyTNErSeOBY0sT5OuCMyr4RsR3YLekY\nSR2kOZE19ToHMzN7tXqOOO4AbpK0GhgBfBZ4Glgi6ZDs84qIeEXSQlIADANmR8ROSYuAZZLWArtJ\nE+KQ7s66Degk3VW1vo7nYGZm++kolUq1e7Ww3t6+oX2CZkPEI5fNanQJQ96J8xfm7tvT09XR33d+\nANDMzApxcJiZWSEODjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEO\nDjMzKyRXcGSLEiLpDyV9QJIDx8ysTdUMAEmfB26QdBSwGvgb4Jv1LszMzJpTnpHDh4GZpGXNl0fE\nDMDv+TYza1N5gqMzInYBHwT+V3aZ6tD6lmVmZs0qT3D8UNIm4BDSpaoHge/XtSozM2taNYMjIv6O\n9ArXKRGxF7g0Iq6oe2VmZtaU8kyOdwNzgX+R9DpgVtZmZmZtKM+lqiXAI8DrgD7gV8DyehZlZmbN\nK09wvDki/iewNyJ2R8Rs4I11rsvMzJpUnuDYI2k8UAKQ9BZgb12rMjOzpjU8R58vAA8AR0n6HvAu\n4Px6FmVmZs2rZnBExD2SHgUmA53AhRHxm2r7SBoBLAWOBkYC84DngJXAM1m3RRFxu6SZwIXAHmBe\nRKyUNJo0j3I4aV7lvIjolTQFWJD1XRURVxU9YTMzOzh57qp6D3BXRPwACOBfJf1xjd3OAZ6PiGnA\n+4GvA5OA6yJievbndklHALOAk4HTgGskjQQuBjZm+98CzMmOu5j0BPtUYLIkP8FuZjbI8sxxzCeN\nCIiIID3TsaDGPt8h3cIL0EEaIUwCPiBptaQbJXUBJwHrImJXRGwDngWOJwXDPdn+dwMzJI0DRkbE\n5ogoAfcCM3Kep5mZDZA8cxyjImJTeSMifpJdiupXRLwIkIXDCtKIYSRwQ0RskDSbNHfyOLCtYtc+\nYDwwrqK9sm37fn0n1Cq+u3sMw4d31upmZjbk9fR0Dchx8gTHTyR9Gbg12/4E8NNaO0k6ErgT+EZE\nfEvSH0TE1uzrO4HrSUuYVJ5JF7CVFBBdVdoq26vasmVHrS5mZm2ht7cvd99qIZPnUtVfAmOBb5Pm\nG8aSVsvtl6TXA6uAKyJiadZ8r6STss+nAhuAh4FpkkZlt/weC2wC1pEuiQGcDqyJiO3AbknHSOog\nzYmsyVG/mZkNoDx3VW0BLil43CuBbmCupPJcx98CX5P0MvBr4IKI2C5pISkAhgGzI2KnpEXAMklr\ngd2kCXGAi4DbSHd3rYqI9QXrMjOzg9RRKpWqdpD0F8BXSUEAabK7FBEtMXHQ29tX/QTNrCk8ctms\nRpcw5J04f2Huvj09XR39fZdnjuPzwPTKCXIzM2tfeeY4/q9Dw8zMyvKMODZIWkGa7N5ZboyIW+pW\nlZmZNa08wTGe9MzEuyraSqQ7rMzMrM3kuavqU5Be6JTdYWVmZm2sZnBIejtwOzAmW2RwNfDxiPhx\nvYszM7Pmk2dy/Hrgo6RFC/8faQHCxXWtyszMmlae4BgTEU+XNyLiPtK6U2Zm1obyBMcL2eWq8hsA\nzwZeqGtVZmbWtPLcVXUxsAz4I0lbSS9iOruuVZmZWdPKExzvjYipkg4FOrPFBs3MrE3lCY6/BhZH\nxO/qXYyZmTW/PMHxnKT7gfXAS+XGiPhS3aoyM7OmlSc4Hqr43O9qiWZm1h7yBMfR5afHzczM8tyO\n+zZJY+teiZmZtYQ8I469wC8lBf9xjuOUulVlZmZNK09wXF73KszMrGXkCQ6/etXMzH4vT3BcVfF5\nBHA8sIa0Sq6ZmbWZPO/jeE/ltqQ3A1+rW0VmZtbU8txV9R9ExM+At9ahFjMzawF5XuR0E/vmOTqA\nY4FNNfYZASwFjiYtwT4PeAq4OTvWJuCSiNgraSZwIbAHmBcRKyWNBpYDh5NeW3teRPRmL5JakPVd\nFRFXYWZmgyrPiOMB4MHsz4+ALwFn1djnHNKLn6YB7we+DlwHzMnaOoAzJR0BzAJOBk4DrpE0krQi\n78as7y3AnOy4i7OfPRWYLGlizvM0M7MBkic47gDGRsQy4IfAFGqPVL4DzM0+d5BGCJNI4QNwNzAD\nOAlYFxG7ImIb8Cxp8n0qcE9lX0njgJERsTkiSsC92THMzGwQ5bmr6jbgyexzHylsbgX+pL8dIuJF\nAEldwArSiOGr2S/88nHGA+OAbRW7Hqi9sm37fn0n1Cq+u3sMw4d31upmZjbk9fR0Dchx8gTHmyLi\nwwDZuzjmSHq81k6SjgTuBL4REd+SdG3F113AVlIQdNVor9W3qi1bdtTqYmbWFnp7+3L3rRYyeS5V\nlSQdV96Q9Fbg5Wo7SHo9sAq4IiKWZs2PSZqefT6d9CzIw8A0SaMkjWffxPs64IzKvllo7ZZ0jKQO\n0pzImhz1m5nZAMoz4vg74D5J/0aarziMNPldzZVANzBXUnmu4zPAQkmHAE8DKyLiFUkLSQEwDJgd\nETslLQKWSVoL7GbfZPxFpEtnnaS7qtbnPVEzMxsYHaVS7RVFsl/2x5FGGptb6W2Avb19XjLFrAU8\nctmsRpcw5J04f2Huvj09Xf2+f6nmpSpJHwc2RMQG4HfATySdmfunm5nZkJJnjmMO2W2vEbGZdFut\nH7wzM2tTeYLjkIj4TXkjIv4dv0LWzKxt5ZkcXyvp26RJaYCPA/9av5LMzKyZ5QmOS4BLSetJ7Sbd\nAfWNehZlZmbNK8+y6rsk3QPsIt0G+0BE7K57ZWZm1pTy3FV1LnAXaaXbNwF3SDq/znWZmVmTynOp\n6jLgpIh4HkDS1aQVc5dW28nMzIamPHdVdZZDAyAifgvsrV9JZmbWzPKMOJ6Q9I/Ajdn2XwJP1K8k\nMzNrZnlGHDNJE+NLSW/w2w38VR1rMjOzJpbnrqqXgCsGoRYzM2sBeUYcZmZmv9dvcEg6dDALMTOz\n1lBtxPEAgCQ/JW5mZr9XbY5jrKTlwPsljdr/y4jwQ4BmZm2oWnC8D3gPMA14cHDKMTOzZtdvcETE\nc8Atkp4AngKU9d8UEXsGqT4zM2syee6qGgE8AywDbgJ+KWlyXasyM7OmlefJ8QXAn0fEegBJU4Dr\ngZPqWZiZmTWnPCOOseXQAIiIh4BXTZabmVl7yBMcL0g6s7wh6SPA81X6m5nZEJbnUtUFwHJJN5Le\nNb4ZOKeuVZmZWdPKs1bVM8Dk7EnyYRHRl/fg2ST6lyNiuqSJwErSRDvAooi4XdJM0mtp9wDzImKl\npNHAcuBwoA84LyJ6s/mVBVnfVRFxVf5TNTOzgZBnxAFARPyuyIElXQ6cC5T3mwRcFxHzK/ocAcwC\nTiDNm6yVdB9wMbAxIr4o6RPAHOAzwGLgT4D/A/xA0sSIeKxIXWZmdnByB8drsBn4GHBrtj0JUDZf\n8gzwWdKdWesiYhewS9KzwPHAVODabL+7gbmSxgEjI2Iz6UD3AjOAqsHR3T2G4cM7B/TEzMxaUU9P\n14Acp2ZwSLooIhYXPXBEfFfS0RVNDwM3RMQGSbOBLwCPA9sq+vQB44FxFe2Vbdv36zuhVh1btuwo\nWrqZ2ZDU25t7pqFqyOS5q+qvc/+k6u6MiA3lz8BEUhBUVtcFbN2v/UBtle1mZjaI8lyqek7S/cB6\n4KVyY0R8qeDPulfSpRHxMHAqsIE0Crk6W0RxJHAssAlYB5yRfX86sCYitkvaLekY0hzHaYAnx83M\nBlme4Hio4nPHQfysi4HrJb0M/Bq4IAuDhcAa0uhndkTslLQIWCZpLelVtWdlx7gIuA3oJN1Vtf5V\nP8XMzOqqo1Qq1eyU3Yp7DGk0MLroHVaN1NvbV/sEzazhHrlsVqNLGPJOnL8wd9+enq5+Bwo15zgk\nnQI8AdwFvB74uaT35f7pZmY2pOSZHL+GdHvs1oj4FfBu4Ct1rcrMzJpWnuAYFhG/Lm9ExFN1rMfM\nzJpcnsnxf5P0QaAk6Q+AS4Bf1rcsMzNrVnlGHBcCZwNHkm6DfQdp4UMzM2tDeRY5/Hfgk9mSHy9H\nxEu19jEzs6Erz5Ijx5FeG3tUtv0T0mq1m+tcm5mZNaE8l6oWkx7MOywiDgPmA0vrW5aZmTWrPMEx\nOiLuLm9ExJ2kBQfNzKwN9XupStJR2ccnJP0DcCPpBUpnk5YIMTOzNlRtjuNBoERan2o66e6qshLp\nBUxmZtZm+g2OiHjzYBZiZmatIc9dVSI9t9Fd2R4R59erKDMza155nhy/E/gn4Mk612JmZi0gT3Bs\nfQ0vbTIzsyEqT3DcLOlq4Ieku6oAiIjVdavKzMyaVp7gmA6cCPxxRVsJOKUeBZmZWXPLExwnRMRb\n6l6J2UH6+5VzGl3CkPeVD85rdAnWBPI8Ob5R0vF1r8TMzFpCnhHHBOAxSb8CdpMeCCxFxIS6VmZm\nZk0pT3B8pO5VmJlZy8gTHO/up/2WgSzEzMxaQ57geE/F5xHANGA1OYJD0mTgyxExXdIfAjeT7sja\nBFwSEXslzSStg7UHmBcRKyWNBpYDhwN9pPd/9EqaAizI+q6KiKtynqeZmQ2QmpPjEfGpij/nABOB\nI2rtJ+ly4AZgVNZ0HTAnIqaR5knOlHQEabHEk4HTgGskjQQuBjZmfW8ByrfLLAbOAqYCkyVNzH+q\nZmY2EPKMOPb3InB0jn6bgY8Bt2bbk0gr7gLcDbwPeAVYFxG7gF2SngWOJwXDtRV952avrh1ZfvOg\npHuBGcBj1Yro7h7D8OGd+c7MzKrq6elqdAl2EAbq7y/PIoc/Il1egjRSmAD8oNZ+EfFdSUdXNHVE\nRPk4fcB40guhtlX0OVB7Zdv2/frWvLNry5YdtbqYWU69vX2NLsEOQpG/v2ohk2fE8cWKzyXgtxHx\nVO6fvs/eis9dwFZSEHTVaK/V18zMBlG/cxySjsreAvizij8/B16seDtgEY9Jmp59Pp30FsGHgWmS\nRkkaDxxLmjhfB5xR2TcitgO7JR0jqYM0J+I3EZqZDbK8bwAsKwFvIN1dVXTi4DJgiaRDgKeBFRHx\niqSFpAAYBsyOiJ2SFgHLJK0lPXR4VnaMi4Dbsp+9KiLWF6zBzMwOUu43AEoaC8wn/Ut/Zp6DR8TP\ngSnZ559ygGdCImIJsGS/th3Anx2g70Pl45mZWWPkWasKSaey70VOx0XEffUryczMmlnVyXFJh5Ke\nvzgNmOnAMDOzapPjpwIbs823OTTMzAyqjzjuA14mPaj3pKRyu1fHNTNrY9WC481VvjMzszZV7a6q\nXwxmIWZm1hpy3VVlZmZW5uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMysEAeH\nmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMyskGpvAKwLST8GtmebPwOu\nBm4GSsAm4JKI2CtpJnAhsAeYFxErJY0GlgOHA33AeRHRO8inYGbW1gZ1xCFpFNAREdOzP58CrgPm\nRMQ00vvMz5R0BDALOBk4DbhG0kjgYmBj1vcWYM5g1m9mZoM/4ng7MEbSquxnXwlMAh7Mvr8beB/w\nCrAuInYBuyQ9CxwPTAWureg7dxBrNzMzBj84dgBfBW4A3kL65d8REaXs+z5gPDAO2Fax34Hay21V\ndXePYfjwzgEp3qzd9fR0NboEOwgD9fc32MHxU+DZLCh+Kul50oijrAvYSpoD6arRXm6rasuWHQNQ\ntpkB9Pb2NboEOwhF/v6qhcxg31V1PjAfQNIbSCOIVZKmZ9+fDqwBHgamSRolaTxwLGnifB1wxn59\nzcxsEA32iONG4GZJa0l3UZ0P/BZYIukQ4GlgRUS8ImkhKRiGAbMjYqekRcCybP/dwFmDXL+ZWdsb\n1OCIiP5+2b/7AH2XAEv2a9sB/Fl9qjMzszz8AKCZmRXi4DAzs0IG/cnxZvaZr3y/0SW0hQV//+FG\nl2BmB8EjDjMzK8TBYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TB\nYWZmhTg4zMysEAeHmZkV4uAwM7NCHBxmZlaIg8PMzApxcJiZWSEODjMzK8TBYWZmhTg4zMyskJZ7\n57ikYcA3gLcDu4BPR8Szja3KzKx9tOKI4yPAqIh4F/APwPwG12Nm1lZaMTimAvcARMRDwAmNLcfM\nrL10lEqlRtdQiKQbgO9GxN3Z9i+BCRGxp7GVmZm1h1YccWwHuiq2hzk0zMwGTysGxzrgDABJU4CN\njS3HzKy9tNxdVcCdwHsl/W+gA/hUg+sxM2srLTfHYWZmjdWKl6rMzKyBHBxmZlaIg8PMzAppxclx\nw0uvDAWSJgNfjojpja7F8pM0AlgKHA2MBOZFxPcbWtQg84ijdXnplRYm6XLgBmBUo2uxws4Bno+I\nacD7ga83uJ5B5+BoXV56pbVtBj7W6CLsNfkOMDf73AG03QPIDo7WNQ7YVrH9iiRfemwREfFd4OVG\n12HFRcSLEdEnqQtYAcxpdE2DzcHRurz0ilmDSDoS+BFwa0R8q9H1DDYHR+vy0itmDSDp9cAq4IqI\nWNroehrBlzZal5deMWuMK4FuYK6k8lzH6RHxUgNrGlRecsTMzArxpSozMyvEwWFmZoU4OMzMrBAH\nh5mZFeLgMDOzQhwc1tYknSDphirff0jS39a5hh/l6PNzSUcP4M+8WdJfDNTxrL34OQ5raxHxKPDp\nKl0mDUIZ0wfhZ5gNGAeHtTVJ04EvZpsPA9OAHuBS4BfARVm/X5AWt/sfwNuATtKS6N/O/uV+HnAY\n8M/AAuCbwJHAXuBzEfEvkk4FrgVKwBbgk8Dns+Ovj4jJOertBL5CCptO4OaI+JqkO4BvRcSKrN+j\nwAWkpWkWAa8DdgCXRsRjxf9Pme3jS1Vm+xySLVP/N6R3LDwFLAYWR8RNpMXsNkTEJOC/A7MlTcj2\nfSMwMSKuJAXH0qzfh4FvZgvizQEuiogTSAHzzoiYBZAnNDIzs/7vBE4CzpQ0DbgV+ASApLcAoyPi\nx8Ay4PKs/wXAP73W/zlmZR5xmO1zT/bfTcB/OsD3M4Axks7Ptg8F/ij7/OOKRSZnAG+V9KVsewRw\nDPB94E5J3wPuioj7XkONM4B3SDol2x4LHEd6t8f1WUB9ErhN0ljgROAmSeX9x0p63Wv4uWa/5+Aw\n22dn9t8Saf2v/XUC52T/ki8vdvcCcDbw0n79TomIF7J+bwB+ExGPS/pn4IPAtZJWRMTVBWvsJI0g\n7siOfRjwu4jYLWklaYTzceADWd+dEfGO8s6S3pjVbPaa+VKVWXV72PcPrPuBiwEk/WfgSeCoA+xz\nP/BXWb//lvUbI2k90BUR/wh8DXhn1r/Iu1TuB2ZKGpGNKNYC5ctctwKXAS9ExC8iYhvwjKRzslre\nC6zO+XPM+uXgMKtuNXC2pEuBq4DRkjaRfoFfHhGbD7DPpcAUSU8CtwPnRkQfaVXVmyVtIM03fCHr\nfxfwhKQ8r5FdDDwDPAY8CtwUEQ8ARMQ6YDywvKL/2cCns1quAf48IryyqR0Ur45rZmaFeI7DrElk\nDwJ2H+CrxRGxeLDrMeuPRxxmZlaI5zjMzKwQB4eZmRXi4DAzs0IcHGZmVoiDw8zMCvn/IYm+TmHe\nrR4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a0974aafd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(train.interest_level);\n",
    "pyplot.xlabel('interest_level');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# drop ids and get labels\n",
    "y_train = train['interest_level']\n",
    "\n",
    "train = train.drop([\"interest_level\"], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'gamma': [0, 0.01, 0.1], 'learning_rate': [0.001, 0.01, 0.1, 1]}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learning_rate = [0.001,0.01,0.1,1]\n",
    "gamma = [0, 0.01, 0.1] \n",
    "param_test6_1 = dict(learning_rate=learning_rate, gamma=gamma)\n",
    "param_test6_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=5, random_state=3, shuffle=True),\n",
       "       error_score='raise',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.8, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
       "       max_depth=6, min_child_weight=0.5, missing=None, n_estimators=192,\n",
       "       n_jobs=1, nthread=4, objective='multi:softprob', random_state=0,\n",
       "       reg_alpha=0.8, reg_lambda=2, scale_pos_weight=1, seed=3,\n",
       "       silent=True, subsample=0.8),\n",
       "       fit_params={}, iid=True, n_jobs=4,\n",
       "       param_grid={'learning_rate': [0.001, 0.01, 0.1, 1], 'gamma': [0, 0.01, 0.1]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#params = {\"objective\": \"multi:softprob\", \"eval_metric\":\"mlogloss\", \"num_class\": 9}\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=192,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=0.5,\n",
    "        gamma=0,\n",
    "        subsample=0.8,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        reg_alpha=0.8,\n",
    "        reg_lambda=2,\n",
    "        objective= 'multi:softprob',\n",
    "        nthread=4,\n",
    "        seed=3)\n",
    "\n",
    "gsearch6_1 = GridSearchCV(xgb1, param_grid = param_test6_1, scoring='neg_log_loss',n_jobs=4, cv=kfold)\n",
    "gsearch6_1.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Anaconda2\\envs\\python3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:667: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.99542, std: 0.00044, params: {'learning_rate': 0.001, 'gamma': 0},\n",
       "  mean: -0.67744, std: 0.00196, params: {'learning_rate': 0.01, 'gamma': 0},\n",
       "  mean: -0.58024, std: 0.00211, params: {'learning_rate': 0.1, 'gamma': 0},\n",
       "  mean: -0.75008, std: 0.00866, params: {'learning_rate': 1, 'gamma': 0},\n",
       "  mean: -0.99542, std: 0.00044, params: {'learning_rate': 0.001, 'gamma': 0.01},\n",
       "  mean: -0.67744, std: 0.00196, params: {'learning_rate': 0.01, 'gamma': 0.01},\n",
       "  mean: -0.58022, std: 0.00202, params: {'learning_rate': 0.1, 'gamma': 0.01},\n",
       "  mean: -0.74854, std: 0.00528, params: {'learning_rate': 1, 'gamma': 0.01},\n",
       "  mean: -0.99542, std: 0.00044, params: {'learning_rate': 0.001, 'gamma': 0.1},\n",
       "  mean: -0.67746, std: 0.00195, params: {'learning_rate': 0.01, 'gamma': 0.1},\n",
       "  mean: -0.57986, std: 0.00236, params: {'learning_rate': 0.1, 'gamma': 0.1},\n",
       "  mean: -0.75231, std: 0.00583, params: {'learning_rate': 1, 'gamma': 0.1}],\n",
       " {'gamma': 0.1, 'learning_rate': 0.1},\n",
       " -0.5798572332848102)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch6_1.grid_scores_, gsearch6_1.best_params_,     gsearch6_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([  159.38418851,   158.38815966,   154.83066978,   147.2238328 ,\n",
       "          149.47262874,   155.81438618,   145.1713779 ,   147.90203815,\n",
       "          148.98912849,  1582.99231477,  1575.73849735,   148.98740859]),\n",
       " 'mean_score_time': array([ 0.66155992,  0.68401999,  0.7471869 ,  0.73655958,  0.6322681 ,\n",
       "         0.68702765,  0.72091851,  0.75079727,  0.62953105,  0.64441442,\n",
       "         0.77415986,  0.79351244]),\n",
       " 'mean_test_score': array([-0.99541584, -0.67743989, -0.58023825, -0.75007566, -0.99541583,\n",
       "        -0.67744032, -0.58022053, -0.74854224, -0.99541506, -0.67745593,\n",
       "        -0.57985723, -0.75230844]),\n",
       " 'mean_train_score': array([-0.99366903, -0.66221874, -0.47384856, -0.12480723, -0.99366906,\n",
       "        -0.66222123, -0.47369087, -0.12329917, -0.99366979, -0.66223805,\n",
       "        -0.47410458, -0.12566054]),\n",
       " 'param_gamma': masked_array(data = [0 0 0 0 0.01 0.01 0.01 0.01 0.1 0.1 0.1 0.1],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'param_learning_rate': masked_array(data = [0.001 0.01 0.1 1 0.001 0.01 0.1 1 0.001 0.01 0.1 1],\n",
       "              mask = [False False False False False False False False False False False False],\n",
       "        fill_value = ?),\n",
       " 'params': ({'gamma': 0, 'learning_rate': 0.001},\n",
       "  {'gamma': 0, 'learning_rate': 0.01},\n",
       "  {'gamma': 0, 'learning_rate': 0.1},\n",
       "  {'gamma': 0, 'learning_rate': 1},\n",
       "  {'gamma': 0.01, 'learning_rate': 0.001},\n",
       "  {'gamma': 0.01, 'learning_rate': 0.01},\n",
       "  {'gamma': 0.01, 'learning_rate': 0.1},\n",
       "  {'gamma': 0.01, 'learning_rate': 1},\n",
       "  {'gamma': 0.1, 'learning_rate': 0.001},\n",
       "  {'gamma': 0.1, 'learning_rate': 0.01},\n",
       "  {'gamma': 0.1, 'learning_rate': 0.1},\n",
       "  {'gamma': 0.1, 'learning_rate': 1}),\n",
       " 'rank_test_score': array([12,  4,  3,  8, 11,  5,  2,  7, 10,  6,  1,  9]),\n",
       " 'split0_test_score': array([-0.99526061, -0.67542876, -0.57699193, -0.74206629, -0.99526073,\n",
       "        -0.67543028, -0.57684236, -0.74979773, -0.99526076, -0.67542892,\n",
       "        -0.57619757, -0.74618093]),\n",
       " 'split0_train_score': array([-0.99379748, -0.66287015, -0.47550008, -0.12310989, -0.99379752,\n",
       "        -0.66287198, -0.47590404, -0.12303077, -0.99379829, -0.66289775,\n",
       "        -0.4753539 , -0.12630259]),\n",
       " 'split1_test_score': array([-0.99508434, -0.6763624 , -0.58069309, -0.76162131, -0.99508434,\n",
       "        -0.67637417, -0.58048624, -0.7520905 , -0.995083  , -0.67640157,\n",
       "        -0.58042722, -0.7627631 ]),\n",
       " 'split1_train_score': array([-0.99380349, -0.66193155, -0.47289206, -0.12632837, -0.99380352,\n",
       "        -0.66194311, -0.47213023, -0.12179753, -0.99380407, -0.66194544,\n",
       "        -0.47334349, -0.12790379]),\n",
       " 'split2_test_score': array([-0.99507156, -0.6761057 , -0.58140304, -0.74675684, -0.99507137,\n",
       "        -0.6760908 , -0.58119379, -0.74440284, -0.99507009, -0.67613692,\n",
       "        -0.58058372, -0.75254334]),\n",
       " 'split2_train_score': array([-0.99378261, -0.66248273, -0.47521376, -0.12592853, -0.99378265,\n",
       "        -0.66248667, -0.47530809, -0.12514359, -0.99378356, -0.66251135,\n",
       "        -0.47519344, -0.12692193]),\n",
       " 'split3_test_score': array([-0.99625814, -0.68074744, -0.58315162, -0.75909955, -0.99625811,\n",
       "        -0.68074803, -0.58298518, -0.7555487 , -0.99625728, -0.68074914,\n",
       "        -0.58335626, -0.75252402]),\n",
       " 'split3_train_score': array([-0.99347348, -0.66134882, -0.47277262, -0.12374587, -0.9934735 ,\n",
       "        -0.66134988, -0.47205439, -0.12383313, -0.99347434, -0.66135803,\n",
       "        -0.47275439, -0.12402258]),\n",
       " 'split4_test_score': array([-0.99540454, -0.6785555 , -0.57895118, -0.74083148, -0.99540459,\n",
       "        -0.67855867, -0.57959488, -0.74086911, -0.99540417, -0.67856344,\n",
       "        -0.57872105, -0.74752935]),\n",
       " 'split4_train_score': array([-0.99348808, -0.66246044, -0.4728643 , -0.12492352, -0.99348812,\n",
       "        -0.66245453, -0.47305763, -0.12269084, -0.99348869, -0.66247769,\n",
       "        -0.4738777 , -0.12315181]),\n",
       " 'std_fit_time': array([  3.73010083e+00,   1.67929910e+00,   6.37967871e-01,\n",
       "          5.03129091e+00,   1.27570198e+00,   1.74850922e+00,\n",
       "          5.23044341e+00,   3.28687180e+00,   2.02859500e+00,\n",
       "          1.75065082e+03,   1.75209127e+03,   2.09051435e+00]),\n",
       " 'std_score_time': array([ 0.04708619,  0.03803501,  0.0509835 ,  0.05395301,  0.00979488,\n",
       "         0.04270652,  0.03140725,  0.05299351,  0.01833823,  0.03613997,\n",
       "         0.03835422,  0.03879529]),\n",
       " 'std_test_score': array([ 0.00043867,  0.00195811,  0.00210938,  0.00866428,  0.00043869,\n",
       "         0.00195911,  0.00202315,  0.00527748,  0.00043877,  0.00195104,\n",
       "         0.00235717,  0.00582737]),\n",
       " 'std_train_score': array([ 0.00015392,  0.00052758,  0.00123552,  0.00123224,  0.00015393,\n",
       "         0.0005263 ,  0.0016142 ,  0.00112996,  0.00015394,  0.0005343 ,\n",
       "         0.00101982,  0.00178953])}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch6_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.579857 using {'learning_rate': 0.1, 'gamma': 0.1}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEFCAYAAADnpECTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4nNWZ9/HvNPVRH3VZkiXrqMvdxriAAYPpEDrhZbOk\nLAnZBLLv7uZNFkgC2ZRNwoYkbCAEyLIpLAk99NiAGy6yituxZavZVrVlq1plZt4/ZiQkYcmSrJmR\nNPfnuriQpt5Hsp7fnPKcx+B0OhFCCCEGGH1dgBBCiOlFgkEIIcQwEgxCCCGGkWAQQggxjASDEEKI\nYcy+LuB8NTe3T3pZVVRUCK2tXVNZzrQnbfYP0mb/cD5tttmshtHu8+seg9ls8nUJXidt9g/SZv/g\nqTb7dTAIIYT4NAkGIYQQw0gwCCGEGEaCQQghxDASDEIIIYaRYBBCCDGMBIMQQohhZvwJbjNNY/UR\n9v/pOULSsyhceznBtjhflySEEMNIMHiR0+nkwJOPkdTUBodqqHv3fU5EhtCTmUX2qktIzCvEYJRO\nnBD+ZtOmD3n22d9gMpm46qprufbaG3xajwSDF3382oskNrVREx9Cc3IitmPHSG7uwryrnI5d5ZQG\nmTiZmkzsgqXkr7oEc3Cwr0sWQnhYf38/jz/+U5566ncEBwdz7733sHLlaqKjY3xWkwSDl/R0dmJ5\n9236jRB2/Z1ctuRCHA4HuysPcOjjvxFcXcmchjZSD9XCoVoOvvgiTfFRGFU+RWuvIDIpxddNEMIj\nXvhbJTsONJ3365hMBux219ZpS3LiuGVt1qiP7ek5w/e+9xAnTjQTFxdPaeluHn74UZ555ikcDgfd\n3d089NAjWCwWHnzwm8THx1NfX88ll6yjquowBw9qVqxYyZe+9BXuu++LZGVlU1V1mODgYIqKFrB9\n+1Y6Ojr46U9/gclk5Ac/eISOjnZaWpq58cZbuOGGm3jnnbfo7u4iP7+Q5ORUwsPDASgqKqa0dDdr\n11563j+TyZJg8JIPf/sL0rr7qchN4TNLLgTAaDSyKDuPRdl5ADScauXjTe/Sd6CMxPomkupboX4T\nTRs3sT8iiI6MdNIuWMO8BctkyEmI8/DKKy+RlJTEI4/8kJqaau666xaqqo7w4IPfIzbWxu9+91s2\nbHiPdevWU19/jJ/97Jf09Jzh5puv4+WX/0pgYBA33XQNX/rSVwDIy8vn61//Jx544KsEBQXx2GO/\n4pFHHqK0tIT4+AQuvXQda9aspaWlmfvu+yI33HAT69ZdAUBZWSlhYWGDtYWEhNLZ2eGTn8sACQYv\nOHr4EMkV+2kPNrL0s18Z9XEJkVFcd/UtcPUt9Pb38/HuHTTs+ojIumqSm7uIKT0ApQcoC/wNTck2\nwooWseCiKwgOs3qxNUJMrVvWZo356X68bDYrzc3t43psTU0Vy5atACAtLZ3IyChsNhuPPfZjgoND\naG5uorCwGIDExGTCwsKwWCxER0cTHh4BgMHwyeak2dk5AFitYaSnZ7i/Dqe3t4fo6GheeOH3fPDB\nBkJCQunv7x9WS2hoKF1dnYPfd3V1DgsKX5Bg8IKDzz1BigNqli1hUXziuJ4TYDazaskFsOQCACrr\naqnY9C7mw/tIaWgl/UgDHHmDqlffoNEWTt+8bPLWrCMlI9uTTRFiVpg7N5M9e8pZvfoijh07yunT\np/jhDx/lhRdeJiQklEceeWjwsUMDYDRjPeaPf3yegoIibrjhJkpKdrJ166Zh96enZ3D0aB1tbacJ\nDg6htHQ3t99+1+QbNwUkGDxsxzuvk3L8JEdtwVx16z2Tfp2s1Dlk3e56fntXF9s2b6C9Yjtxx+tJ\nbmyDxp10bdrJ1vAAWtNSiF98AcXLLsJstkxVU4SYNa6++joeffQ7fOUrXyAhIYGAgAAuv3w9X/7y\nFwgODiIqKoaWluYpea8LL1zNz372I95//x3CwsIwmUz09vaycePf6O7u4rrrbuS+++7ngQe+isPh\n4KqrrsXm42XsBqdz0te5mTClVDDwPBAHtAN3a62bRzxmPfAQYAB2AV/RWo9a5PlcqGciXc/J6Os5\nw+5/+SrhnX00/Z+7WL3qkil/D4fTScXePRzZ+j6h1YdJaWnHbHfd1xVopCExBnNeAQsvWk90dJzH\n2zwdSZv9w0TaXFFRRnd3N0uXLqeurpZvfOOrvPDCKx6ucOqdz+95rAv1eLvHcC9QobV+WCl1G/Bt\n4GsDdyqlrMCPgYu01i1KqX8GYoGpiW4v2/jcE6R19LEvK57rPRAKAEaDgeKCQooLCgFoaj7Bro1v\nYdflJDW0MLe6Gao30PjWBkptofTMyyJt2RpycxaMq4ssxGyUlJTMww9/i2eeeZL+/n4eeOBffF3S\ntOLtYFgJ/Mj99ZvAv424fwVQAfxEKTUX+M3IHsVM0XS0lsSScroCjRTdea/X3jfOFsP6m+8E7qS3\nt49d2zbTXLKZ6KN1pDR2QmMZbCpjW7iFlpQEwosXs2TlZYQEhnitRiF8LSYmlscf/7Wvy5i2PDaU\npJS6B7h/xM2NwH1a6/1KKSNQq7VOGfKcO4GfAPOBDuAj4Fat9cHR3qe/3+6cjpf0++PXvkxqdSP7\nVy3k8//0LV+XA8CBfQfZ9dZrGA/uI7HpNBb3mu/uAAPHkyIx5hVwwfpryJyT6eNKhRBe4P2hJK31\n08DTQ29TSv0FGFhbaQVOjXjaCWCH1rrB/fgPcYXEqMFwPhf/9tQ4bOmHfyO1upGGqEAuu/WL02as\nN8aWyJ0PfIPm5nba29op2fgenXt2En+8kczqVqj+iGNvfsROWzCn56SSuGwli4tXYDbO7DUKMt7u\nH6TNE3/uaLz9F78ZuBLYDqzH1SMYqgQoUErF4gqN5cBTXq3wPNn7+uh++U+EAL2XX0NIUJCvSzor\na7iVNdfeANfegN3hYP+OXdRu30h4TRUpTV2kNB2EnQfZGf476pNiCcwvZunKdcRafXeavhDCO7wd\nDE8AzymlNgG9wB0ASqkHgEqt9atKqW8Cb7sf/4LWeo+XazwvG/7nN8xp60Gnx3DNpVf7upxxMRmN\nFCxbQsGyJQA0HT1OxYa34eAeEppOEn2gAQ40cPy1d/g4wUr33EzUiovJzyjAaJAzsIWYbby6XNUT\nptNy1ZONDRx9+JsYgMB/+ibZmdPvZLOJtrmnu4fyDzZysnQrsceOYe3uA8BhgOO2QFpSEomav5Tl\ni9dgDQj1VNnnRYYY/MNMbvO5dlc9deoU3/nOt+jp6SE21sb/+38PERQUhM1mpa6umfvv/zL/+q8P\nkpaWPu73nE7LVWe1Hb99nIw+J3sW53LjNAyFyQgMDmTJFZfDFZfjcDioqdjH4c3vE1R1iOSmDlKa\nqqGkmn3/+2eOJkVBdh7FK9aSGZsmy2GFGIfx7K767LNPcdllV3Dlldfw3//9LK+88mduvfVOKioq\n+Na3/o3m5vPfhHAoCYYpsmf7FtIPH6Ml3MKld93n63I8wmg0klFcQEZxAQBtjSfYs+EdzuwrJa6x\niULdAvpDut76iDcSQ2hLSyN16SqWZi8i0BTg4+rFdPWXytfZ3VRx3q9jMhqwO1wDCAviCrkxa/Sh\n3Jm2u2p5eSl33fU5AJYvX8GTT/6SW2+9k97eXr7//R/zve89eN4/v6EkGKaAvb+f1hefJx5oW7uO\n8NDpOaQy1cLjY1hx2+3A7dh7ezmweSsNOzYRWVdDdm0n1O7DsWkfm2MtNCbHEZQ/n+WL15BolavW\nCd+aaburdnZ+srFeSEgIHR2u+xctWuSR4TMJhinwwYv/TcrJLg6nRHD5lZ/xdTk+YQoIIP/iNeRf\nvAan00mTPoz+8D1MlftIbm4jpfkYlB6j7pW32JQURm9mNnlL11CYnDPjl8OK83Nj1tVjfrofr9m8\nu6rrMV0EBgbR1dWF1erZHZXlL/I8tbW2Ev3RR/SZIPWWezDKdRIwGAzE52QRn+PaSvnMyVb2bdxI\nW/kOYusbKD54Gg7uoOe9nbyVEMiJ1CRi51/AyrxlRASF+7h64Q9m2u6qhYXFbN26mSuvvIZt27ZQ\nVDR/gi2eGAmG87T56f8ks8fB3uIsbsgr8nU501JQdBQLb7wBbrwBe28vtTt2U7vtQ0JrKsmuOwN1\nR3BsPUJJ7AscS4rEqApYtHAV82LSZTms8IiZsLtqW9tpfvCDR/j+93/M3XffwyOPPMxrr71EREQk\nDz306JTUNhpZrnoe43O6vATn4z+nLcRM1nf/g+iIyEm/lrdMpyV9TqeTtuoaDmzcgP1AOdEnWhmI\ngdOhRqqSQ2hPyyBj4YUsTy8mxDK5a2BPpzZ7i7R5bLK7qixX9Qin00n9H54hyQktq1ezdAaEwnRj\nMBiIyEhnWYZrtUXf6TYqP9pMS8lWIo8fY/7BDjhYQc+GPWxIDKAhOQ5rwUIuyFtOijVBlsOKSZPd\nVccmPYZJpu0HL/2BxDfepiYhjEu++/MZM7cwUz5JOvv7adxdQdWWDwg4orF2drtuB+pjzdQmWenN\nVBQXr6A4MYeAMZbDzpQ2TyVps3+QHsM00tXZTtjf3qPfCDGf+T8zJhRmEoPZTMKSBSQsWYDT6aSz\n7iiVH2yie98uEppbSGpphfJtnH5nOy8nBXEyNZnEgqVcmLWQ2BDZz0mI8yHBMAkfPP04md129uXO\n4foFS31dzqxnMBgIm5PK/Ltc50z0d7RTt2U7DTu2EFZXzfxDXXDoEL0fVrI98UWOJkViVoUsy11G\nTsxcX5cvxIwjwTBBVQf3k7bnIO3BJlb83Vd9XY5fModZyVh3CRnrLsHZ38/Jvfup3vQRHNpLVl0n\nWXWNOD9upCFmI88nhdCdmUlmzmKWzSkiPMCz67+FmA0kGCbo8PO/Zo4Djl+wjEUxNl+X4/cMZjMx\nxYXEFLsubXqmvp7qDzdzunwn8U0NJJ5og4rdtIWU8XZyII3JNiJyFnBh1kLSI1JlOawQZyHBMAGb\n33yZOcdPcSw2mCtu/XtflyPOIigxkZxbb4Jbb8Le2UnD9l007dpG4OFDFB/qhkO19H1Ui058k3cS\nw+jPzGFhzhIWJOQQbJ7cclghzte5dlcd8MILv+fEiRPce69nRyskGMapp7sLy9tv4DBAyPW3YzHJ\nj266M4WGknzxaubfchVNDadoO6Cp3bSFvgNlZB5tJ/PoSdixhYbo7byQHMiJ1GTS1CJWpBWTEBov\ny2GFV4xnd9WenjP84AePsH//XtasWevxmuToNk7vP/srsjr60FmJXLN8ta/LERNkMJmIyM+jMD8P\ngJ6Geo5v2c6J3duJazhOwslOqDhI+4ZKtiS/RF1SJIFZBazIXEhu7DwCTBYft2D2av7fP9K+c8d5\nv06NyYjd7gDAungJtptvG/WxM2131Z6eXtavv5olS5ZRU1N93j+rc5FgGIe6miPMKdtLV6CRhXfP\nzi21/U1gQiIZN15Hxo3XYe/q5GRJGce2bSXgyAEKK89QWNlA3+YG6hI+ZEtSMB3pGRRmLmRpSiHR\nQVG+Ll+cp5m2u2p4eDhLly7nr399zSs/HwmGcdj33K/I6HdStWwh8xOTfV2OmGKmkFBsK1dgW7kC\np91OZ+Uhjm7+mN49u5l77BRzj/XCjlIao/fwenIg9cmx2OYWsTJ9PpmR6ZiMJl83YUaz3XzbmJ/u\nx/06s3h3VW+TYDiHbRvfJqO2hcaoQNbd+UVflyM8zGAyEaZyyFE5wN30NjXR9PEOmnduJ/Z4LfEn\nO6Gik47gOiqS3uX15FBIVyxOn8+ChDysAb79gxbjM9N2V/U2CYYx9PX24njtLwAYrrqRwIBAH1ck\nvC0gLo6Ua64i5ZqrsHd3c7qsjONbPybg0D4KD5+h8PAZ+jdvoS5+J88nB3IyJZnMtEIumFNEqjVZ\nlsNOUzNtd1Vvk72Sxuh6vvnbx8ncsovKdBtXftv7vxxPkP1kpobT4aD7cCX1W7fTUV5C8KmTg/c1\nRZmpSg6gLikca1oeF6QVkx+rCDYHTWkNY5Hf89hkd1XZK2lSGo4fI3lnCT0WA3mfvdfX5YhpxmA0\nEjIvm8x52cBn6WtupmXnLpp37CC27ghxrV0s29NFZ1ATVclb+CApiO6UNObPKWZJUgFxITZZDutD\nsrvq2CQYRlHy3C/I6nVyYEkB16bLfjtibBabjcT1V5C4/gocZ7ppr9hDw7btWA5UUHD4DAWHz9Bv\nPMXR+P28lBJAfVIMKcn5rJhTRHZUJhZZDutVMTGxPP74r31dxrQlwXAWu7Z+SObhek6EW7js7i/7\nuhwxwxiDgolYsoSIJUtwOhycqTpC0/ad9OzeRXp9M+n1vUAHzZHH2J28kZeSQjAlZ7I0pYiFCflE\nBcm1PYRvSTCMYO/vp/OlP2AFei67iuCgEF+XJGYwg9FIcGYWaZlZcPtt9J1o4VRJCc3bdxJTU4nt\nVBdL93bRFXSCqqQynkkO5GRiArmJeSxLKSQjIk2Wwwqvk2AY4d0/PcPck91UJUdx+frrfV2OmGUs\nMbHYLluH7bJ1OM6coXPfXhq37cCyr5z8I13kHzlDv/E0R+OreD/pXeqSrcTEZ7M8pYhCWw5hAaG+\nboLwAxIMQ5xoaSZ+y1b6TJBxxxd8XY6Y5YxBQVgXLsK6cJFryKm6mpM7d3GqZBfp9Q2uIaddHbRE\nNnEkaQfvJQfSH5/K/MR8FicVkBKWJBPYwiMkGIbY+uzjZPc4OFCczbUqz9flCD9iMBoJnjuX5Llz\nSb7lZvpOnqC9tJSmj3cSXXWQ2FNdLNnXRVfgKaqTDvJi8hs0JEaSEZvDstRCcqPnEeTF5bBidpNg\ncKso2U6mruV0qJmL5QI8wscs0TFEr72E6LWX4OjpoWv/Ppq37ySgopS8qk7yqs5gN7RxNL6eHclb\n+N+kIEJjMlicXMCC+DxsyAWJxORJMAB2u53mF58jxQmnL74Eq1X+qMT0YQwMJGz+AsLmL8DpcNBT\nW8OpkhJad5WQ1nCMtIY+2NXBiYhWqpL38mRyAB1xNnJjcliaUkhW5FwsRvlTF+Mn/1qA91/6PelN\nndTGW7n4mlt8XY4QozIYjQSlZ5CQnkHCjZ+hr7WVjvJSWrbvJKpSE7Ovi8X7uugOOE11Uh1vJ3/A\nbxJCSIzKZGlKIUW2XCIDI3zdDDHNSTAAwR9vxm6AxFs/h8kkSwPFzGGJiiJqzcVErbnYNeR0YD+t\nO3fhqCgjt7qN3GrXkNOxuBPo5DLeSg7AEJnA/Lg8FiUVkB4+R/ZzEp8iwQAE9vbTHWhiYdFCX5ci\nxKQZAwMJK55PWPF8YmPDOLprD22lpZzcuYs59XXMaexjTQmcDD/NkeRq/pD8Hi2xVjIj57E0pYD8\nmBxCLXLejpBgAMDodOKQD01iFjEYDASlpROUlk7cddfTf+oUneVlnNi5i0i9j8X7u1i8v4vugDZq\nkprYmrSTPyQGEmlNYVFiPvPj8kgOS5TlsH5KggEwOqDfJH8AYvYyR0YSsXoNEavX4OjtpevAfk6X\nlOAoLSWn+jQ51T04DHDMdoqq5EM8kfxXeiIiyY1WLE4qQEVlEWSWbef9hQQDAz0G6TII/2AMCCCs\nqJiwomKcTic9dbV0lJVycmcJqcdqSG3qY/VuOGltoyq5njeSt/J0bCBJYeksSSqgyJaHLSTm3G8k\nZiwJBsDocOIwSo9B+B+DwUDQnDSC5qQRe8119J8+RWdFOa27Sog8sI9FB7pZdKCbMxYjNUmn2Ju8\nnzeSAggIjqHIlsvChHyyIjMwy3LYWUV+m4DRCU4ZSxUCc0QkEStXE7FyNY6+Xrr1Adp2l+LcvRtV\n04qqcQ05HY9toyrlGP+d/CHt1iDmhmeyJKmAgthcIgLDfd0McZ4kGHD3GCQYhBjGaAkgtKCI0IIi\nEj57F71Hj9JRVkprSQnJtVWkNPeyaje0hpqpSmnlw+QK/hhnISowgQUJecyPyyMtPFWWw85AEgy4\negwylCTE6AwGA4GpqQSmphJz9TX0t7XRWVFG2+7dOPfuYaHuZqHupsdspCaxjSMpNWxKeh9HUAg5\nUYpFifnkRWcTIsthZwSvBoNSKhh4HogD2oG7tdbNQ+6fDzw25CnLgeu11m95qia73S5DSUJMkDk8\nnIgLVxFx4SocfX10H9R0lJVyuqSE7LqTZNf14ADqYwKoSj3BK8m7eCbcTFJwKosS8ym05ZIUmiDL\nYacpb/cY7gUqtNYPK6VuA74NfG3gTq11KXARgFLqZuCYJ0MBoLe3B0BWJQkxSUaLhdD8AkLzC4i7\n/U56jx+ns2w3p0pKSKquIvlELytLOzkVYqEqpZ3SlMO8YXuTIEs4hbG5LEjIQ0VlEWAK8HVThJu3\ng2El8CP3128C/3a2BymlQoHvAKvP9YJRUSGYzZPfxsJqdf1jdBqN2Gz+sXmev7RzKGmzF8WFw/wc\nuPt2+traaN1ZQsv2HVBSyoKD3Sw42E2PyUhNYjtVKU08l7SNniALWVGZXJg+n0VJhcSFxU7qreX3\nPDU8FgxKqXuA+0fc3Aicdn/dDoy2m9c9wP9qrVvO9T6trV2TrtFms1Jf3wqAwwDNze2Tfq2Zwmaz\n+kU7h5I2+5IBQ+EibIWLiL27n+5DB+ko3c3p3bvJPtpC9tEzOGmnPjqQqtQ2Xq3axzMRJiItMcyP\nz6c4LpfMiIxxXd50+rTZe86nzWMFiseCQWv9NPD00NuUUn+BwY3ircCpUZ5+J3CTp2obqq/3DODq\nMQghPMdgNhOSm0dIbh622+6gt76ezrJS2kp3k3ikkqSTPVxY1snpYAtVyZ0cSa3no7gPMZgCyY7M\nYmFCPnkxOUQE+l+vwNu8PZS0GbgS2A6sBz4a+QClVAQQqLWu80ZB9v4+QCafhfAmg8FAYFISgUlJ\nRK+/EntHB50V5XSUlUJFBfMru5hfCb0mIzXxQVTN2cWfk/bwfJCRuMBEFibkUWjLZY41RZbDeoC3\ng+EJ4Dml1CagF7gDQCn1AFCptX4VyAaqvVVQX58rGBzj6KoKITzDFBZG+AUrCL9gBc7+frorD9FR\nVkrb7hLmHW9m3nFcQ06RQVSldrEzpY63It8jyBRKfoyiOC6PVRGyO/JUMTidTl/XcF6am9sn3QCb\nzcqmD7dg+MlPqEy3ceW3fzyVpU1LMg7rH2ZTm3sb6ukoK6W9tJQzlYcwOB0AtAVZqEoOpCrVxNH4\nABwmI6khc1iYmE9+TA6JofGzfjnsec4xjPrD8fsT3Pr7+rAAyByDENNSQEIi0QmJRF++3jXktLfC\nNeRUXk7x4Q6KD0Ov0UhtXDBH5hzgnRNHeDn4r4SZwimOy6PIlkt2VBYBJouvmzJjSDD09WIBnDJO\nKcS0ZwoLI3zZBYQvu8A15HS4kk53byKroYGsBtfj6iOCqUrpRs85webIrRgNZjLD57IwIZ/8mFxi\ngqN825BpToLBPcfglEt6CjGjGMxmQlQOISoH2y23EdbXTt2GLbSX7Sbx0EES93azYi+0BQZQlRhI\nddoeXjx5kD+ZXyYmIJb58fkUxuYwNyJ9XMth/YnfB4Pd3u/6QnoMQsxowUlJRK27nKh1l2Pv6qRr\nzx46ykoxlJdRXN1OcTX0GY3UxIZSPecoW0838n7dB1gMgeRGz6M4Lo/8mBysAWG+borPSTAMLFc1\nSTAIMVuYQkKxLl2GdekynHb74JBTR1kpWQ31ZDUBO6HBGkJVSg9Vabspb6kAg4Gk4CTmx+dREJtL\nqjXZL5fD+n0wOPrdPQbpSgoxKxlMJkKyFSHZCtvNt9Lb1ERnuSskEvQBEvZ3ccF+aA8IoCohhKq0\nat5pP8Zfq98j2BhKoS2HQlsuudHZBJuDfN0cr5BgcPcYZFWSEP4hIC6OgEvXEXXpOuxdXXTt+2TI\nqaj2FEW10G80UhMdRvWcXvbO2c72xl0YMJJuTXP1JmJyiA+Jm7XLYf0+GOwDPQaZfBbC75hCQrAu\nXop18VKcDgdnDh+mo2w3HeVlZB4/RmYLUAINYaFUJwVSlXGQl9qO8FLlG4RbIpkf5xpyyo6ci2UW\nLYf1+2Bw2O0AGKTHIIRfMxiNBM+bR/C8edhuuoXe5iY6y8voHBhyOtjJ8oPQYQmgKj6UqrRmtnRv\n5sNjWzBhJjsqc3ACOzpoZi+H9ftgsMscgxDiLAJscQRcchlRl1yG40w3nXv30FlWhrG8jMKjrRQe\nhX6DkdooK1WpdqrT97G/VQMQFxRHsbs3kRE+Z8Yth/X7YHC6l6saTH7/oxBCjMIYFIx10RKsi5a4\nhpyqjgyucpp77ChzTwJl0BQSypGkYKoyjvNudyPv1m4kwBhIfoyiMDaXvBg1I5bD+v3R0DFwHoMs\nVxVCjIPBaCQ4M4vgzCxib7yJvpZmOsvL6CgrJe7AAeIqO1leCZ2WAI7EWqlK62ZPTym7m8sBSA1N\npTgul/zYHFLDkqflBLbfB8Ngj8Ho9z8KIcQkWGJtRK69lMi1l+I4c4bOfXvpLC/FVFZGYf0JCuvB\n/rGR2shwjiQHUD23htc763i96h1CzWEU2XIpiMkhJ3oeQdNkOazfHw2d/e7J5/O4PKgQQgAYg4Kw\nLlyEdeEi15BTdRWd5aV0lpWSUVdHRiuwB5pDwjgcH0pNRhvbereztX4HRoxkRmRQZMslPyaHuBCb\nz3oT4woGpVSi1rpeKbUKKAKe1Vp3erY073A6BlYlSTAIIaaOwWgkeG4mwXMzib3+M/SdPEFnmWvI\nyXZgH7aqDpZXQZc5kCMx4VTNger0Qxw6fZg/V75OVECUawI7JpesyAyvLoc9ZzAopZ4AHEqpXwK/\nB94B1gKf8XBt3uFermo0z541yEKI6ccSHUPkxWuJvHgtjp4euvbtpaO8FFN5GQWNzRQ0gn2nkbrw\nCI4kBVGdeZKNvZvZeHQzZoOFnOgsCmNdvYmooEiP1jqeHsNSYDHwEPC01vphpdQOj1blRc6B8xhk\nVZIQwkuMgYGELVhI2IKFOB0Oempr6ChzDTml19aQfhrYDy3BVg7bwqhOd7DXvo89J/YDkBAcT3Fc\nPrdGXOmR+sZzNDQBRuA64B+UUiFAqEeq8QWHa/LZKGc+CyF8wGA0EpSeQVB6BrHX3UBfa+vgvIRh\n/z5ia9vhiBNnAAASQElEQVRZVgtnzIEcjorkSIqFurkNvN3diMUYwvqM1VNe03iC4XdAPbBZa/2x\nUmo/8F9TXomPOB2uywSazNJjEEL4niUqisg1FxO55mLXkNP+fXSWl2EqLyW/uZH8ZnCUGqkLiybh\ns7GQMfU1nPNoqLX+qVLqP7XWdvdNK7XWJ6a+FB+RYBBCTFPGwEDC5i8gbP4C4pxOemprXCfWlZeR\nVl1FlsUza4DGM/l8NbBKKfU9YAdgU0o9pLX+pUcq8jaZYxBCzAAGg4GgtHSC0tKJufZ6HH29xCVG\n09LSMeXvNZ7TfR8CngFuA7YD6cDnprwSX3EvVzVZZFWSEGLmMFoCPHaew7j2gdBaHwCuAl7VWncA\nAR6pxgcMg0NJEgxCCAHjC4ZGpdTjwBLgLaXUT4Baz5blRQPBIENJQggBjC8Ybsc1t7DGfbbzEfdt\ns4LBPcdglqEkIYQAxrdctQMIA36olDIDG4BZsR0GgMEpQ0lCCDHUeILhR8A84LeAAdfEcwbwdQ/W\n5T3uoSRLwKyZNhFCiPMynmBYByzQWjsAlFJvABUercqLDA4nABaLBIMQQsD45hjMDA8QM2Af5bEz\njlGGkoQQYpjx9Bj+B9iolPqD+/vbgT+M8fgZZWC5qkUmn4UQAhjflhjfV0rtxrXVthF4VGv9hscr\n85LBoaSA6XHlJCGE8LVxLd7XWr8JvDnwvVLqV1rrL3usKi8aWJUkPQYhhHAZ15nPZ/HZKa3Ch4zu\nHkNgoEw+CyEETD4YfHMhUg8wOF3BICe4CSGEy2SDwTmlVfiQ0eHAYQCLrEoSQghgjDkGpdQGzh4A\nBiDYYxV5mdHhxDFr+j9CCHH+xpp8fthbRfiSwenEYZRkEEKIAaMGg9b6A28W4itGp/QYhBBiqMnO\nMcwaRof0GIQQYiivXoRAKRUMPA/EAe3A3Vrr5hGP+QZwB+AAvq+1fsmTNUmPQQghhhvPNZ9Xj7jJ\nCXQDlVrrUxN8v3uBCq31w0qp24BvA18b8l6R7u+zgFCgFPBsMEiPQQghhhnPUNKDwKu4DthfB14B\nngR2KqUmesGelcBb7q/fBC4dcX8nUIMrFEJx9Ro8yugAh4eumyqEEDPReIaSDECR1roWQCmVBDwD\nXARsZJQN9ZRS9wD3j7i5ETjt/rodiDjLU+uAfYAJ+PdzFRcVFYLZbDrXw0ZldDpxGo3YbNZJv8ZM\n409tHSBt9g/S5qkxnmBIGggFAK31caVUota6TSk16kdtrfXTwNNDb1NK/QUYaIUVGDkUtR5IxHUh\nIIC3lVKbtdbbR3uf1taucTTh7Gw2q3uOwUBzc/ukX2cmsdmsftPWAdJm/yBtnvhzRzOeYNislPo9\nru23jcBtwFal1FW4Lvs5EZuBK4HtuELgoxH3t+Kav+jRWjuVUqeAyAm+x4QYHMgcgxBCDDGeYPgH\n939fBPqB94CncF3Z7a4Jvt8TwHNKqU1AL67VRyilHsA1mf2qUupSYJtSygFsAt6d4HtMiMndYxBC\nCOEynusx9CulNuKaazABW7XW/cBfJ/pmWusu4Oaz3P7TIV8/BDw00deeLKP0GIQQYphzrkpSSt2F\nayVSOpAG/EUp9fcerssr7HY7Ric4pccghBCDxjOU9A1gqdb6BIBS6lFcq5F+68G6vKK3txeQ5apC\nCDHUeM5jMA2EAoDWugUvnF/gDWfOnAHAYfT7nUGEEGLQeHoMZUqpx/hk6ek9QJnnSvKePnePAZlj\nEEKIQeP5qPwFoAfX0NGzuFYT3evBmrymZ3AoSXoMQggxYDyrkrqBfxl6m3srjLOe8TyT9Pa4hpKc\n0mMQQohBk/2o/OsprcJH+vr6AHDKHIMQQgya7BFxVnzE7u3pASQYhBBiqMkeEc92LegZR3oMQgjx\naaPOMSilHhzlLgMQ4JlyvKvfPfnslMlnIYQYNNbk81jDRefcDnsm6O9zB4P0GIQQYtCowaC1/s7I\n25RSV2utX/dsSd7T32d3/QCMk7+egxBCzDYT/aj8XY9U4SP9fTL5LIQQI030iDgrViMN6HdPPiPB\nIIQQgyZ6RHzVI1X4iN3uDgaTDCUJIcSACQWD+1oJs4a9t9/1hfQYhBBikF8fEe390mMQQoiR/DoY\nHHY7AAbpMQghxCC/PiI6BiafTePZfVwIIfyDXweD3e6aYzDIUJIQQgzy62BwDgSDnOAmhBCD/DoY\nHP3SYxBCiJH8OxgGJp9ljkEIIQb5dTDgHkoySo9BCCEG+XUwOKXHIIQQnyLBABjNFh9XIoQQ04cE\nAzKUJIQQQ/l1MCA9BiGE+BT/DgaHAwCTWeYYhBBigH8Hg7vHYDJJj0EIIQb4dzA43MFgkWAQQogB\nfh4M7qEkmXwWQohB/h0MdncwBAT4uBAhhJg+/DoYDO6hJLOsShJCiEF+HgyuHoNZViUJIcQgvw4G\nHE4AzAGBPi5ECCGmD78OBoPT1WOwWGSOQQghBvh3MAwMJclyVSGEGOTXwWB0B0OABIMQQgzy62Aw\nOF1zDJaAIB9XIoQQ04dXl+MopYKB54E4oB24W2vdPOIx/wLcDrQBP9Jav+6pegzSYxBCiE/xdo/h\nXqBCa70K+B3w7aF3KqUKgTuA5cA64LtKqRBPFWN0r0qyBMrksxBCDPD2Av6VwI/cX78J/NuI+3OB\njVrrMwBKqUNAEbBttBeMigrBbJ7clhYGpxOHAZISoyf1/JnKZrP6ugSvkzb7B2nz1PBYMCil7gHu\nH3FzI3Da/XU7EDHi/grgm0opKxAArACeHOt9Wlu7Jl2j0eHAYYDm5vZJv8ZMY7NZ/aq9IG32F9Lm\niT93NB4LBq3108DTQ29TSv0FGKjGCpwa8Zz9SqlfAG8BtcDHQIunajQ4nDiMBk+9vBBCzEjenmPY\nDFzp/no98NHQO5VSNsCqtb4Q+AcgFdjjqWKMTidOyQUhhBjG23MMTwDPKaU2Ab24JppRSj0AVAKv\nAblKqR3u+/+v1truqWKMDid26TEIIcQwXg0GrXUXcPNZbv/pkG+/5K16jO7JZyGEEJ/w7xPcHOA0\nSDIIIcRQfh0MJqdMPgshxEh+HQwGCQYhhPgUvw4Go8OJQ4aShBBiGD8PBiQYhBBiBP8OBhlKEkKI\nT/HvYJBVSUII8Sl+Gwx2ux2TE+kxCCHECH4bDP39fYD0GIQQYiS/DYbenl4AHEa//REIIcRZ+e1R\nsbevBwCnDCUJIcQwfhsMfb2uoSSHwW9/BEIIcVZ+e1TsG5hjkB6DEEIM47/B0HsGAKfMMQghxDB+\ne1Ts7+sHwClDSUIIMYzfHhX7ZShJCCHOym+Dwd43EAwmH1cihBDTi98Gw0CPAZljEEKIYfz2qNjf\n5zrBTSafhRBiOL89Ktr73ZPPMpQkhBDD+HEwyFCSEEKcjd8eFQd6DBIMQggxnN8eFZ32gWCQoSQh\nhBjKb4PB4R5KMpgkGIQQYii/DYbBoSST3/4IhBDirPz2qOiw211fyFCSEEIM47fB4HT3GGQoSQgh\nhvPfYHC4egwGk9nHlQghxPTit8HgcK9KMshQkhBCDOO3wTCwXFV6DEIIMZzfBgN2BwBGs/QYhBBi\nKL8Nhk96DBIMQggxlN8GA+7JZ6PJ4uNChBBievHbYHAODiXJHIMQQgzlt8HwSY9BgkEIIYby32Bw\nn/lssshQkhBCDOW/weB0DyXJ5LMQQgzjt8FgcLiCwWSWHoMQQgzlt8EwMJRklmAQQohhfDLzqpS6\nAbhZa33HWe77AvAloB94RGv9uidqMDidAJhkVZIQQgzj9R6DUuo/gX8/23srpRKAfwQuBC4H/l0p\nFeiJOqKWruJIRhw5hQs88fJCCDFj+WIoaQtw7yj3LQU2a617tNangUqgyBNFrLj4Cu567AmCgoI9\n8fJCCDFjeWwcRSl1D3D/iJs/p7X+k1LqolGeFg6cHvJ9OxAx1vtERYVgPo/9jmw266SfO1NJm/2D\ntNk/eKLNHgsGrfXTwNMTfFobMLSVVuDUWE9obe2a4Ft8wmaz0tzcPunnz0TSZv8gbfYP59PmsQJl\nus28bgceVUoFAYFALrDHtyUJIYR/mRbBoJR6AKjUWr+qlPo58BGu+Y9vaa3P+LY6IYTwLwane9nm\nTNXc3D7pBkjX0z9Im/2DtHnCzzWMdp//nuAmhBDirCQYhBBCDCPBIIQQYpgZP8cghBBiakmPQQgh\nxDASDEIIIYaRYBBCCDGMBIMQQohhJBiEEEIMI8EghBBiGAkGIYQQw0yLTfQ8SSllBH4FFAM9wOe1\n1pVD7r8GeBDXpUR/q7V+yieFTqFxtPl24Ou42lwBfFlr7fBFrVPlXG0e8rgngZNa63/1colTbhy/\n5yXATwED0AB8dqZvSjmONt8JfAOw4/p7fsInhXqAUmoZ8EOt9UUjbp/yY5g/9BiuB4K01hcA/wr8\nZOAOpZQF+BmwDlgDfFEpFe+TKqfWWG0OBh4BLtZaX4jrQkhX+6TKqTVqmwcopb4EFHq7MA8a6/ds\nAJ7CdXGslcBbQJpPqpxa5/o9/wdwKa7LA39DKRXl5fo8Qin1z8BvgKARt3vkGOYPwTDwR4HWehuw\neMh9ubi2+27VWvcCm4DV3i9xyo3V5h5ghdZ64ApHZmBGf4p0G6vNKKVWAMuAX3u/NI8Zq83ZwAng\nfqXUB0C01lp7v8QpN+bvGSjH9WEnCFdPabZs7XAYuPEst3vkGOYPwTDycqF2pZR5lPvOeSnRGWLU\nNmutHVrrRgCl1FeBMOBd75c45UZts1IqEXgIuM8XhXnQWP+2Y4EVwC9wfYK+RCm11sv1ecJYbQbX\nhb12AXuB17XWY14BcqbQWv8Z6DvLXR45hvlDMIy8XKhRa90/yn3nvJToDDFWm1FKGZVS/wFcBnxG\naz0bPlWN1eabcR0o/4pr+OEOpdTfebc8jxirzSdwfZLcr7Xuw/Upe+Sn65lo1DYrpYqAq4AMIB2I\nU0rd7PUKvcsjxzB/CIbNwJUASqnluCZbB+wH5imlopVSAbi6YFu9X+KUG6vN4BpOCQKuHzKkNNON\n2mat9c+11ovck3Y/AH6vtX7WF0VOsbF+z0eAMKVUlvv7Vbg+Rc90Y7X5NNANdGut7UATMCvmGMbg\nkWPYrN9ddcgqhiJcY46fAxYCYVrrJ4fM6Btxzej/0mfFTpGx2gzsdP/3EZ+Mv/6n1volH5Q6Zc71\nex7yuL8DcmbZqqTR/m2vxRWEBmCL1vprPit2ioyjzf8A/D3Qi2tc/gvusfcZTymVDvxRa71cKXUH\nHjyGzfpgEEIIMTH+MJQkhBBiAiQYhBBCDCPBIIQQYhgJBiGEEMNIMAghhBhGgkHMakqpi5RSGz34\n+t9VSl3rqdc/x3t/Rym1yhfvLWa3Wb+7qhCepLV+0IdvvwbY4MP3F7OUBIPwC+4zgJ8AYoAu4Kta\n691KqQLgcVwn/8UBP9Fa/1wp9TCwHJiDa7+hW4DtuM4gtrmf/6ZS6llgo/u/l3Dt1bMAaARu1lqf\nVErdAnzX/b4lgFlr/Xdj1LoROAnkA7fi2jjuLiAUcLhvW4Jri4vfKKVuwHXG76fadx4/MuHHZChJ\n+IvngH/WWi8Evgj80X3754FHtNZLgIuBR4c8J0hrnae1/pX7+wD3ds/349q6fKRi4Kda6wJc+9Xc\nqZSyAY8Bl+A6kEePs95yrbXCtbXF9cBF7td9Gdf1M36H6wz2z2utK8ZonxATJj0G4Q/CcG2h8IxS\navA2pVQMrou6XKGU+qb7MWFDnvfxiNd5y/3/PZz9AN805FP6wGNWAVu11scAlFLPATeMo+aPAbTW\nbe7tD25TSmUDVwClQx+olArD1YP4VPu01ifG8V5CDCPBIPyBCTijtZ4/cINSKgXXcM2LQCvwGq5P\n2bcNeV73iNcZuG6FE9c+PSMNva7FwGPsTK5n3u2uMxXXMNUvgDdxXYltwYjHjtU+ISZMhpKEPzgN\nHFJKfRZAKXUZ8KH7vsuAB7XWr+CazEUpZZrC994CLFFKJbqvqnYbE7t4zBJc22f/DFcvYj2uIADX\npRzNWuux2ifEhEkwCH9xJ/B5pVQ58O/Are7rUDwMbFJKlQCXA9W49vOfElrrZuAfcV0MaQdg4dM9\nkbG8AxiVUvuAbSPqewv4L/fV6UZrnxATJrurCuFB7nmMfwS+o7V2KKV+DhzSWj/u49KEGJXMMQjh\nWSeBSGCPUqof13LVp5RS/4NrOepIr/r43AghpMcghBBiOJljEEIIMYwEgxBCiGEkGIQQQgwjwSCE\nEGIYCQYhhBDD/H9ZdReiFvJAkgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2a0aa7099b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch6_1.best_score_, gsearch6_1.best_params_))\n",
    "test_means = gsearch6_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch6_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch6_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch6_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch6_1.cv_results_).to_csv('my_preds_gamma_vs_learning_rate_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(gamma), len(learning_rate))\n",
    "\n",
    "    \n",
    "for i, value in enumerate(gamma):\n",
    "    pyplot.plot(learning_rate, test_scores[i], label= 'gamma:' + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'learning_rate' )                                                                                                      \n",
    "pyplot.ylabel( '-Log Loss' )\n",
    "pyplot.savefig( 'reg_alpha_vs_reg_lambda1.png' )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 结论默认的learning_rate=0.1和gamma=0就OK"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
